package com.chance.cc.crawler.development.bootstrap.weibo.api;

import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.apache.commons.lang3.StringUtils;
import org.junit.Test;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;

/**
 * @author lt
 * @version 1.0
 * @date 2021-03-15 10:32:41
 * @email okprog@sina.com
 */
public class WeiboApiStart {
    public static final String domain = "weibo";
    public static final String site = "apikw";
    public static final String kwSite = "medical_keyword";
    public static final String authorSite = "medical_weibo_uid";
    public static final String midSite = "medical_weibo_mid";

    public static final String siteBiz = "article-realtime";
    /*
    是否采集Age、Tag、Interaction
     */
    public static final String IS_OUTPUT_Age = "is_output_age";
    public static final String IS_OUTPUT_Tags = "is_output_Tags";
    public static final String IS_OUTPUT_Interaction = "is_output_Interaction";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        //HL89Q19E86E2987D
        //71F33D94CE5F7BF2
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static void main(String[] args) {

        CrawlerRequestRecord searchKWRecord = searchKWRecord();
        CrawlerRequestRecord supportRecord = getKwRecord(kwSite);

        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .triggerInfo(domain, domain + "_trigger", System.currentTimeMillis(), domain + "_job")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain)) //内存队列
                .consoleResultPipeline() //控制台输t出
                .fileResultPipeline("D:\\chance\\data\\weibo\\weibo_article_04-01_data.json", false)
                .requestRecord(searchKWRecord)  //more job
                .supportRecord(supportRecord)
                .crawlerThreadNum(10)
                .build();
        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addKVTag(IS_OUTPUT_Age,true);
        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addKVTag(IS_OUTPUT_Tags,true);
//        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addKVTag(IS_OUTPUT_Interaction,true);
        devCrawlerController.start();
    }

    public static CrawlerRequestRecord searchKWRecord(){
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, turnPageItem)
                .httpUrl("https://www.weibo.com")
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*10,null))
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-","filter",domain,site,"queue")))
                .needWashed(false)
                .needParsed(false)
                .proxy(proxy)
                .build();
        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);
        return requestRecord;
    }

    public static CrawlerRequestRecord getKwRecord(String metaSite){
        //keyword record
        return CrawlerRequestRecord.builder()
                .startPageRequest(StringUtils.joinWith("-",domain, site),turnPageItem)
                .httpUrl("http://192.168.1.215:9599/v1/meta/"+domain+"/keys?site=" + metaSite)
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();
    }


    @Test
    public void testDealer(){
        HttpConfig httpConfig = HttpConfig.me("test");
//        httpConfig.setProxy(proxy);
//        httpConfig.setUseProxy(true);

        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());

        String url = "https://c.api.weibo.com/2/search/statuses/limited.json?access_token=2.00o4_w1HrAaeYBedf38e38b8SnITmD&q=%E9%95%81%E4%BF%A1&count=50&endtime=1615965147&page=1";
        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setUrl(url);
        HttpPage httpPage = downloader.download(httpRequest, httpConfig);
        System.out.println(httpPage.getRawText());
        JSONObject jsonObject = JSONObject.parseObject(httpPage.getRawText());
    }
}
